/****************************************************************************
 *
 * Copyright (C) 2003-2008 Los Alamos National Security, LLC
 *                         Packet Analytics Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License Version 2 as
 * published by the Free Software Foundation.  You may not use, modify or
 * distribute this program under any other version of the GNU General
 * Public License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 ****************************************************************************/
package nfse.index;

import java.util.*;
import java.io.*;

import nfse.*;

/**
 * @author bduphoff
 */
public class IndexBuilder {

    public static int build(IndexBuildInput input) {
        try {
            int numDBs = input.fields.length;

            int val = 0;

            MetaDatabase mdb = MetaDatabase.create();

            GlobalField[] fields = new GlobalField[input.fields.length];
            for (int i = 0; i < input.fields.length; i++) {
                fields[i] = NetFSE.getGlobalField(input.fields[i].dbName);
            }

            String result = input.globalDir +"/"+ input.toolName + "/" + input.sensorName + "/" + input.year + "/"
                    + input.month + "/" + input.day + "_" + input.run + "." + input.toolName;

            String[] dataFilePaths = new String[input.nodes.length];
            if (input.buildInPlace) {
                dataFilePaths = new String[1];
                dataFilePaths[0] = result;
            }
            for (int i = 0; i < numDBs; i++) {
                String[] toMerge = new String[input.nodes.length];
                for (int j = 0; j < toMerge.length; j++) {
                    toMerge[j] = input.sharedDir + "/" +input.nodes[j] + "/" + input.toolName + "/" + input.sensorName
                            + "/index/" + input.year + "/" + input.month + "/" + input.day + "_" + input.run + "/"
                            + input.fields[i].dbName + "/index.dump";
                    if (!input.buildInPlace) {
                        dataFilePaths[j] = input.sharedDir + "/" +input.nodes[j] + "/" + input.toolName + "/"
                                + input.sensorName + "/" + input.year + "/" + input.month + "/" + input.day + "_"
                                + input.run + "." +  input.toolName;
                    }
                }
                if (input.buildInPlace) {
                    dataFilePaths = new String[1];
                    dataFilePaths[0] = result;
                }
                val = performIndexMerge(toMerge, dataFilePaths, input.fields[i].dbPath + "/index.data",
                        fields[i].getLength());
                if (val < 0)
                    throw (new Exception("Index merge failure."));
            }

            if (!input.buildInPlace) {
                int fileMergeVal = performFileMerge(dataFilePaths, result);
                if (fileMergeVal < 0) {
                    throw (new Exception("Failed to merge file, val=" + fileMergeVal));
                }
            }

            // System.out.println("Merge complete.");
            input.dataFiles = new String[1];
            input.dataFiles[0] = result;

            for (int i = 0; i < numDBs; i++) {
                IndexBuilderParms parms = new IndexBuilderParms();
                parms.setDataFilePath(input.dataFiles[0]);
                parms.setIndexFilePath(input.fields[i].dbPath + "/index.data");
                parms.setResultDir(input.fields[i].dbPath);
                parms.setKeyLength(fields[i].getLength()); 
                parms.setPageSize(input.pageSize);
                parms.setMonth(input.month);
                parms.setDay(input.day);
                parms.setYear(input.year);
                parms.setRun(input.run);
                parms.setType(input.dataType);
                parms.setField(fields[i].getId());
                val = buildIndex(parms);
            }

            if (val >= 0) {
                for (int i = 0; i < numDBs; i++) {
                    IndexInfo info = new IndexInfo();
                    info.setSensorId(input.sensorID);
                    info.setField(input.fields[i].id);
                    info.setMonth(input.month);
                    info.setDay(input.day);
                    info.setYear(input.year);
                    info.setRun(input.run);
                    info.setDbPath(input.fields[i].dbPath);
                    info.setMinTime(input.minTime);
                    info.setMaxTime(input.maxTime);
                    mdb.addStaticIndex(info);
                }
                DataInfo info = new DataInfo();
                info.setSensor(input.sensorID);
                info.setMonth(input.month);
                info.setDay(input.day);
                info.setYear(input.year);
                info.setRun(input.run);
                info.setMinTime(input.minTime);
                info.setMaxTime(input.maxTime);
                info.setDataPath(input.dataFiles[0]);
                info.setNumBytes(input.bytes);
                info.setNumRecords(input.records);

                if (input.buildInPlace)
                    mdb.removeData(input.month, input.day, input.year, input.run, input.sensorID);
                mdb.addData(info);

            }
            mdb.closeConnection();

            return val;
        } catch (Exception E) {
            E.printStackTrace();
        }
        return -1;
    }

    public static void main(String[] args) {
        try {
            File inputFile = new File(args[0]);
            if (!inputFile.exists()) {
                System.out.println("Usage: java IndexBuilder input_file_path");
                return;
            }
            build(IndexBuildInput.processParms(args[0]));
        } catch (Exception E) {
            E.printStackTrace();
        }
    }

    private static int performFileMerge(String[] dataFiles, String resultPath) {
        try {
            long size = 0;
            int val = 0;
            for (int i = 0; i < dataFiles.length; i++) {
                File file = new File(dataFiles[i]);
                DataInputStream in = new DataInputStream(new FileInputStream(dataFiles[i]));
                DataOutputStream out = null;
                size += file.length();
                if (i == 0) {
                    out = new DataOutputStream(new FileOutputStream(resultPath));

                } else {
                    out = new DataOutputStream(new FileOutputStream(resultPath, true));

                }
                int numRead = 1;
                byte bytes[] = new byte[1024];
                while (numRead > 0) {
                    numRead = in.read(bytes, 0, 1024);
                    if (numRead > 0)
                        out.write(bytes, 0, numRead);
                }
                in.close();
                out.close();
            }
            
            return val;
        } catch (Exception E) {
            E.printStackTrace();
        }
        return -1;
    }

    private static int performIndexMerge(String[] inputFilePaths, String[] dataFilePaths, String resultPath,
            int fieldLength) {
        try {
            int numFiles = inputFilePaths.length;
            if (numFiles != dataFilePaths.length) {
                System.err.println("File parameters incorrect. Sizes must be equal.");
                return -1;
            }
            long[] oafs = new long[numFiles]; // Offset adjustment factors
            long sumFileLengths = 0;

            LineNumberReader[] lnrs = new LineNumberReader[numFiles];
            for (int i = 0; i < numFiles; i++) {
                File file = new File(dataFilePaths[i]);
                if (!file.exists()) {
                    String errorMsg = "The file '" + dataFilePaths[i]
                            + "' could not be opened. Possible disk or NFS error.";
                   throw (new Exception(errorMsg));
                }

                oafs[i] = sumFileLengths;
                sumFileLengths += file.length();
                lnrs[i] = new LineNumberReader(new FileReader(inputFilePaths[i]));
            }
            PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(resultPath)));

            String[] valueBuffer = new String[numFiles];
            long[] offsetBuffer = new long[numFiles];
            for (int i = 0; i < numFiles; i++) {
                valueBuffer[i] = "";
                offsetBuffer[i] = -1;
            }

            long count = 0;
            String lastValue = "";
            long minOffset = -1;
            // long offset = -1;
            String minValue = "";
            String line = "";
            boolean done = false;
            boolean firstValue = true;
            while (!done) {
                // Load/reload the buffer with the next line
                // and find min value
                int minIdx = -1;
                minValue = "";
                for (int i = 0; i < numFiles; i++) {
                    if ((valueBuffer[i].length() == 0) && (line != null)) {
                        line = lnrs[i].readLine();
                        // System.out.println("LINE: '" + line + "'");
                        if ((line != null) && (line.length() >= 8 + fieldLength)) {
                            offsetBuffer[i] = Long.parseLong(line.substring(0, 8), 16);
                            offsetBuffer[i] += oafs[i];
                            valueBuffer[i] = line.substring(8, 8 + fieldLength);
                        }

                    }
                    if (valueBuffer[i].length() > 0) {
                        // Check to see if this is the minValue

                        if ((minIdx == -1) || (valueBuffer[i].compareTo(minValue) < 0)) {
                            minIdx = i;
                            minValue = valueBuffer[i];
                            minOffset = offsetBuffer[i];
                        }
                    }
                }
                if (minIdx == -1) // The buffer is empty
                    done = true;
                else { // Write the min value
                    /*
                     * cout < < util.hexToShort((char
                     * *)minValue.substr(0,2).c_str()) < < "." < <
                     * util.hexToShort((char *)minValue.substr(2,2).c_str()) < <
                     * "." < < util.hexToShort((char
                     * *)minValue.substr(4,2).c_str()) < < "." < <
                     * util.hexToShort((char *)minValue.substr(6,2).c_str()) < < ", " < <
                     * util.hexToLong((char *)minValue.substr(8,8).c_str()) < <
                     * endl;
                     */
                    // string value = minValue.substr(0, keyLength);
                    // string offset = minValue.substr(keyLength, keyLength +
                    // 8);
                    if ((lastValue.length() == 0) || (!lastValue.equals(minValue))) {
                        if (!firstValue) {
                            out.println();
                            // System.out.println();
                        }
                        out.print(minValue);
                        // System.out.print(minValue);
                        lastValue = minValue;
                        firstValue = false;
                    }
                    String offsetStr = "" + Long.toHexString(minOffset);
                    while (offsetStr.length() < 8)
                        offsetStr = "0" + offsetStr;
                    out.print(offsetStr);
                    // System.out.print(offsetStr);
                    valueBuffer[minIdx] = "";
                    offsetBuffer[minIdx] = -1;
                    count++;
                }
            }

            for (int i = 0; i < numFiles; i++) {
                lnrs[i].close();
            }
            out.close();
            return 0;
        } catch (Exception E) {
            E.printStackTrace();
        }
        return -1;
    }

    private static int buildIndex(IndexBuilderParms parms) {
        try {

            String infoPath = parms.getResultDir() + "/info";
            PrintWriter infoOut = new PrintWriter(new BufferedWriter(new FileWriter(infoPath)));

            infoOut.println("dataFile " + parms.getDataFilePath());

            // Check key length: length > 0 indicates fixed length, length of -1
            // indicates a delimited field
            if (parms.getKeyLength() > 0)
                infoOut.println("keyLength " + parms.getKeyLength());
            else if (parms.getKeyLength() == -1)
                infoOut.println("delimiter " + parms.getDelimiter());
            infoOut.println("pageSize " + parms.getPageSize());
            infoOut.println("month " + parms.getMonth());
            infoOut.println("day " + parms.getDay());
            infoOut.println("year " + parms.getYear());
            infoOut.println("run " + parms.getRun());
            infoOut.println("type " + parms.getType()); // The key type (i.e.
            // integer, long, hex int,
            // string, etc)
            infoOut.println("file 1"); // The file ID in the metadata;
            // depreciated
            infoOut.println("field " + parms.getField()); // The field ID in the
            // metadata
            infoOut.close();
            return buildIndexTree(parms);
        } catch (Exception E) {
            E.printStackTrace();
        }
        return -1;
    } /*
         * Method: buildIndexTree Description: Build the index tree structure to
         * allow the index to be searched.
         * ---------------------------------------------------- Paramaeters:
         * None. Returns: 0 - Success Negative on error
         * 
         */

    private static int buildIndexTree(IndexBuilderParms parms) {

        IndexPage[] pages = buildLeafPages(parms);
        if (pages == null) {
            System.err.println("Failed to create leaf pages.");
            return -1;
        } else if (pages.length == 0) {
            System.err.println("Failed to create leaf pages.");
            return -2;
        }
        // System.out.println("Built leaf pages.");

        int val = buildPages(parms, pages, 1);
        if (val < 0) {
            System.err.println("Failed to create interior pages.");
            return val;
        }
        // System.out.println("Built interior pages.");
        return 0;
    } /*
         * Method: buildLeafPages Description: Build the leaf pages of the tree.
         * ---------------------------------------------------- Paramaeters:
         * None. Returns: 0 - Success Negative on error
         * 
         */

    private static IndexPage[] buildLeafPages(IndexBuilderParms parms) {
        try {
            String indexFile = parms.getResultDir() + "/index.data";
            File file = new File(indexFile);
            if (!file.exists()) {
                System.err.println("Failed to open '" + indexFile + "' for input.");
                return null;
            }
            RandomAccessFile in = new RandomAccessFile(indexFile, "r");
            Vector<IndexPage> resultPages = new Vector<IndexPage>();

            Vector<String> values = new Vector<String>();
            Vector<Long> offsets = new Vector<Long>();
            int pageCount = 0;
            int delimLen = 0;
            int delimOffset = 0;
            String delimiter = parms.getDelimiter();
            if (delimiter.length() > 0)
                delimLen = delimiter.length();
            int ch = 0;
            while (ch != -1) {
                boolean delimFound = false;
                long offset = in.getFilePointer();
                String temp = "";

                int count = 0;
                if (parms.getKeyLength() > 0) { // Fixed length key
                    ch = in.read();
                    while ((ch != -1) && (count < parms.getKeyLength())) {
                        temp += (char) ch;
                        count++;
                        ch = in.read();
                    }
                } else { // Variable length key
                    while ((ch != -1) && (delimOffset < delimLen)) {
                        ch = in.read();
                        if ((char) ch == delimiter.charAt(delimOffset)) {
                            delimOffset++;
                        } else if (delimOffset > 0) { // Case where part of
                            // the
                            // delim
                            // is seen already but a char
                            // doesn't match
                            for (int x = 0; x < delimOffset; x++)
                                temp += delimiter.charAt(x);
                            delimOffset = 0;
                            temp += (char) ch;
                        } else {
                            temp += (char) ch;
                        }
                    }
                    if (delimOffset == delimLen) {
                        delimOffset = 0;
                        delimFound = true;
                    }
                }
                String value = temp;
                if ((parms.getKeyLength() > 0) && (value.length() != parms.getKeyLength())) {
                    System.err.println("buildLeafPages: Bad key length.");
                } else if ((parms.getKeyLength() == -1) && (!delimFound)) {
                    System.err.println("Key delimiter not found.");
                } else {
                    values.addElement(value);
                    offsets.addElement(new Long(offset));
                    if (values.size() == parms.getPageSize()) {
                        String pagePath = parms.getResultDir() + "/pages/page" + (++pageCount) + ".page";
                        PrintWriter pageOut = new PrintWriter(new BufferedWriter(new FileWriter(pagePath)));
                        for (int i = 0; i < values.size(); i++) {
                            pageOut.print((String) values.elementAt(i));
                            if (parms.getKeyLength() == -1)
                                pageOut.print(delimiter);
                            String offsetStr = Long.toHexString(((Long) offsets.elementAt(i)).longValue());
                            while (offsetStr.length() < 8)
                                offsetStr = "0" + offsetStr;
                            pageOut.println(offsetStr);
                        }
                        pageOut.close();
                        IndexPage page = new IndexPage();
                        page.pageID = pageCount;
                        page.pagePath = pagePath;
                        resultPages.addElement(page);
                        values = new Vector<String>();
                        offsets = new Vector<Long>();
                    }
                }
                if (ch != -1) {
                    while (((char) ch != '\n') && (ch != -1)) {
                        ch = in.read();
                    }
                }

            } // End While
            if (values.size() > 0) { // Write out any left over value/offset
                // pairs
                // as a new page
                String pagePath = parms.getResultDir() + "/pages/page" + (++pageCount) + ".page";
                PrintWriter pageOut = new PrintWriter(new BufferedWriter(new FileWriter(pagePath)));
                for (int i = 0; i < values.size(); i++) {
                    // pageOut << values[i] << setw(8) << setfill('0') <<
                    // hex << offsets[i] << endl;
                    pageOut.print((String) values.elementAt(i));
                    if (parms.getKeyLength() == -1)
                        pageOut.print(delimiter);
                    String offsetStr = Long.toHexString(((Long) offsets.elementAt(i)).longValue());
                    while (offsetStr.length() < 8)
                        offsetStr = "0" + offsetStr;
                    pageOut.println(offsetStr);
                }
                pageOut.close();
                IndexPage page = new IndexPage();
                page.pageID = pageCount;
                page.pagePath = pagePath;
                resultPages.addElement(page);
            }

            // Update file index
            String fileIndexPath = parms.getResultDir() + "/file.index";
            PrintWriter fileIndex = new PrintWriter(new BufferedWriter(new FileWriter(fileIndexPath)));
            for (int i = 0; i < pageCount; i++) {
                String str = Integer.toHexString(i + 1);
                while (str.length() < 4)
                    str = "0" + str;
                str += ((IndexPage) resultPages.elementAt(i)).pagePath;
                fileIndex.println(str);
            }
            fileIndex.close();
            in.close();
            IndexPage[] temp = new IndexPage[resultPages.size()];
            for (int i = 0; i < temp.length; i++)
                temp[i] = (IndexPage) resultPages.elementAt(i);
            return temp;
        } catch (Exception E) {
            E.printStackTrace();
        }
        return null;
    }

    /*
     * Method: buildPages Description: Build the non-leaf pages of the tree.
     * Recursive. ----------------------------------------------------
     * Paramaeters: pages - file paths to build this level on. pageIDs - IDs for
     * each page. level - what level this will be in the tree. Returns: 0 -
     * Success Negative on error
     * 
     */
    private static int buildPages(IndexBuilderParms parms, IndexPage[] pages, int level) {
        /*
         * if (level > 1) { cout < < "Building on:" < < endl; for (int i = 0; i <
         * pages.size(); i++) { cout < < pageIDs[i] < < ": '" < < pages[i] < <
         * "'" < < endl; } }
         */
        try {
            if (pages.length == 0)
                throw (new Exception("No more pages error, this should not happen!"));
            Vector<IndexPage> resultPages = new Vector<IndexPage>();
            int numPages = pages.length;
            int nextPageID = pages[pages.length - 1].pageID + 1;
            Vector<String> values = new Vector<String>();
            for (int i = 0; i < numPages; i++) {
                LineNumberReader in = new LineNumberReader(new FileReader(pages[i].pagePath));
                String line = in.readLine();
                in.close();
                String value = "";
                if (parms.getKeyLength() != -1)
                    value = line.substring(0, parms.getKeyLength());
                else {
                    int delimLoc = line.indexOf(parms.getDelimiter());
                    if (delimLoc > 0) {
                        value = line.substring(0, delimLoc);
                    } else {
                        System.err.println("Bad page data, delimiter not found.");
                        return -1;
                    }
                }
                String temp = value;
                if (parms.getKeyLength() == -1)
                    temp += parms.getDelimiter();
                String idStr = "" + Integer.toHexString(pages[i].pageID);
                while (idStr.length() < 4)
                    idStr = "0" + idStr;
                values.addElement(temp + idStr);
                if (values.size() == parms.getPageSize()) {
                    String pagePath = parms.getResultDir() + "/index/index" + level + "_" + resultPages.size() + 1 + ".page";
                    PrintWriter pageOut = new PrintWriter(new BufferedWriter(new FileWriter(pagePath)));
                    for (int x = 0; x < values.size(); x++)
                        pageOut.println((String) values.elementAt(x));
                    pageOut.close();
                    IndexPage page = new IndexPage();
                    page.pageID = nextPageID++;
                    page.pagePath = pagePath;
                    resultPages.addElement(page);
                    values = new Vector<String>();
                }
            }

            if (values.size() > 0) {
                String pagePath = parms.getResultDir() + "/index/index" + level + "_" + resultPages.size() + 1 + ".page";
                PrintWriter pageOut = new PrintWriter(new BufferedWriter(new FileWriter(pagePath)));
                for (int i = 0; i < values.size(); i++)
                    pageOut.println((String) values.elementAt(i));
                pageOut.close();
                IndexPage page = new IndexPage();
                page.pageID = nextPageID++;
                page.pagePath = pagePath;
                resultPages.addElement(page);
            }

            if (resultPages.size() == 1) {
                String rootPath = parms.getResultDir() + "/index/root.page";
                File root = new File(((IndexPage) resultPages.elementAt(0)).pagePath);
                File newFile = new File(rootPath);
                root.renameTo(newFile);
                ((IndexPage) resultPages.elementAt(0)).pagePath = rootPath;
            }

            // Write results to file index
            String fileIndexPath = parms.getResultDir() + "/file.index";
            PrintWriter fileIndexOut = new PrintWriter(new BufferedWriter(new FileWriter(fileIndexPath, true)));
            for (int i = 0; i < resultPages.size(); i++) {
                IndexPage page = (IndexPage) resultPages.elementAt(i);
                String temp = "" + page.pageID;
                while (temp.length() < 4)
                    temp = "0" + temp;
                temp += page.pagePath;
                fileIndexOut.println(temp);
            }
            fileIndexOut.close();

            if (resultPages.size() > 1) {
                IndexPage[] temp = new IndexPage[resultPages.size()];
                for (int i = 0; i < temp.length; i++)
                    temp[i] = (IndexPage) resultPages.elementAt(i);
                buildPages(parms, temp, ++level);
            }

            return 0;
        } catch (Exception E) {
            E.printStackTrace();
        }
        return -1;
    }
}